#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
from contextlib import closing
try:
    from bs4 import BeautifulSoup
except ImportError:
    raise ImportError,"The Python BeautifulSoup module is required to run this program. Try 'pip install beautifulsoup4'."
try:
    import requests
except ImportError:
    raise ImportError,"The Python requests module is required to run this program. Try 'pip install requests'."

def list_hrefs(url, pattern=None, case_insensitive=True):
    """Scrape a web page for hrefs, filter by regex pattern"""
    if pattern:
        rx = re.compile(pattern, re.I) if case_insensitive else re.compile(pattern)
    with closing(requests.get(url)) as r:
        html = r.text
    soup = BeautifulSoup(html)
    hrefs = []
    for link in soup.find_all('a'):
        href =  link.get('href')
        if href and (pattern == None or rx.match(href)):
            hrefs.append(href)
    return hrefs

url = 'http://evs.gs.washington.edu/EVS/' #http://evs.gs.washington.edu/evs_bulk_data/ESP6500SI-V2-SSA137.protein-hgvs-update.snps_indels.vcf.tar.gz
hrefs = list_hrefs(url, r'.*evs_bulk_data.*\.vcf.*')
print hrefs


done_file = os.path.join(nhlbi_dir,'.nhlbi.done')
if my.file_exists(done_file):
    self.print_log('NHLBI-EVS already installed. To reinstall "rm {}"'.format(done_file))
else:
    self.print_log('Installing NHLBI-EVS.')
    default_nhlbi_vcf_file_url = None
    if not nhlbi_vcf_file_url:
        try:
            default_nhlbi_vcf_file_url = config.get('nhlbi', 'DEFAULT_NHLBI_VCF_FILE_URL')
        except:
            pass
        #determine version if not specified as a run parameter
        self.print_log('Getting list of NHLBI-EVS VCF file urls.')
        nhlbi_info_url = config.get('nhlbi', 'NHLBI_INFO_URL')
        nhlbi_vcf_file_pattern = config.get('nhlbi', 'NHLBI_VCF_FILE_PATTERN')
        nhlbi_vcf_file_urls = list(set(self.list_hrefs(nhlbi_info_url, nhlbi_vcf_file_pattern)))
        if not nhlbi_vcf_file_urls:
            self.print_log('No urls at {} match the pattern {}. Check the web site and NHLBI_VCF_FILE_PATTERN in vax.cfg.'.format(nhlbi_info_url, nhlbi_vcf_file_pattern))
        else:
            if len(nhlbi_vcf_file_urls) == 1:
                nhlbi_vcf_file_url = nhlbi_vcf_file_urls[0]
            else:
                nhlbi_vcf_file_url = self.select_item(nhlbi_vcf_file_urls, default_nhlbi_vcf_file_url, 'NHLBI-EVS VCF files', 'NHLBI-EVS VCF file to install')
    if not nhlbi_vcf_file_url:
        self.print_log('WARNING: No NHLBI-EVS VCF file selected. Rerun or install manually.')
    else:
        #download selected version
        nhlbi_vcf_file = os.path.join(nhlbi_dir, os.path.basename(nhlbi_vcf_file_url))
        self.print_log('Downloading NHLBI-EVS VCF file {} to {}.'.format(nhlbi_vcf_file_url, nhlbi_vcf_file))
        urlretrieve(nhlbi_vcf_file_url, nhlbi_vcf_file)
        self.print_log('NHLBI-EVS VCF downloaded.')
        #remember version as default
        print_log('setting NHLBI-EVS VCF file url {} as default for next install.'.format(nhlbi_vcf_file_url))
        config.set('nhlbi', 'DEFAULT_NHLBI_VCF_FILE_URL', nhlbi_vcf_file_url)
        
        #process VCF file to create data for mySQL table
        
        
        with open(done_file, 'w'):
            pass
        self.print_log('Installed NHLBI-EVS.')


